/**
 * 
 */
package edu.umd.clip.lm.programs;

import java.io.*;
import java.nio.channels.*;

import edu.berkeley.nlp.util.*;

import edu.umd.clip.lm.model.*;
import edu.umd.clip.lm.model.data.*;
import edu.umd.clip.lm.ngram.*;
import edu.umd.clip.lm.util.IO;

/**
 * @author Denis Filimonov <den@cs.umd.edu>
 *
 */
public class Ngram {
	public static class Options {
        @Option(name = "-config", required = true, usage = "XML config file")
		public String config;
        @Option(name = "-forest-file", required = false, usage = "Output LM file (Default: stdout)")
		public String lmFile;
        @Option(name = "-jobs", usage = "number of concurrent jobs (default: 1)")
        public int jobs = 1;
        @Option(name = "-debug", usage = "decoder debug level (default: 0)")
        public int debug = 0;
        @Option(name = "-ppl-file", required = true, usage = "Training data file for perplexity computation")
		public String pplFile;
	}

	/**
	 * @param args
	 * @throws IOException 
	 * @throws ClassNotFoundException 
	 */
	public static void main(String[] args) throws IOException, ClassNotFoundException {
        OptionParser optParser = new OptionParser(Options.class);
        final Options opts = (Options) optParser.parse(args, true);
		
        Experiment.initialize(opts.config);
		Experiment experiment = Experiment.getInstance();
		
		experiment.buildPrefixes();
		experiment.buildWordPrefixes();

		ObjectInputStream ois = new ObjectInputStream(IO.getInputStream(opts.lmFile));
		NgramModel model = (NgramModel) ois.readObject();
		ois.close();
		
		ReadableByteChannel channel = new FileInputStream(opts.pplFile).getChannel();
		TrainingDataReader reader = new OnDiskTrainingDataReader(channel);
		ReadableTrainingData inputData = new ReadableTrainingData(reader);
		
		double totalLogProb = 0;
		long totalWordCount = 0;
		while(inputData.hasNext()) {
			TrainingDataBlock block = inputData.next();
			Pair<Double,Long> pair = model.computeTrainingDataLogprob(block);
			
			totalLogProb += pair.getFirst();
			totalWordCount += pair.getSecond();
		}

		channel.close();
		
		System.out.printf("Total word count: %d, log10 prob = %g, ppl = %g\n", 
				totalWordCount, totalLogProb, Math.pow(10.0, -totalLogProb/totalWordCount));
	}

}
